from selenium import webdriver
#from scrapy.http import HtmlResponse
import time
from selenium import webdriver
from selenium.webdriver.common import action_chains
import time
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.action_chains import ActionChains
from lxml import etree
from scrapy.middleware import MiddlewareManager
#from scrapy.downloadermiddlewares.
from scrapy.http.response.html import HtmlResponse
import pymysql
from scrapy.selector import Selector

# class JSMiddleware():
#     # 定义鼠标滚动到页面最底端的js函数
#     def scroll(self,driver):
#         driver.execute_script("""
#             (function () {
#                 var y = document.body.scrollTop;
#                 var step = 100;
#                 window.scroll(0, y);
#                 function f() {
#                     if (y < document.body.scrollHeight) {
#                         y += step;
#                         window.scroll(0, y);
#                         setTimeout(f, 50);
#                     }
#                     else {
#                         window.scroll(0, y);
#                         document.title += "scroll-done";
#                     }
#                 }
#                 setTimeout(f, 1000);
#             })();
#             """)
#
#     """
#     注意: 由于京东的页面滚动到最下面的时候,其实是加载了两页的内容
#           所以,page=1和page=2其实是一样的内容
#           文胸分类上点击下一页其实是跳转到page+2的url
#           page_num取值范围[1,400]之间的偶数或者奇数
#     """
#     from scrapy.http import HtmlResponse
#
#     def get_bralist_by_page(self,page_num):
#         driver = webdriver.PhantomJS(executable_path="D:\\phantomjs-1.9.7-windows\\phantomjs.exe")
#         driver.set_page_load_timeout(30)
#         driver.get("https://search.jd.com/Search?keyword=%E6%96%87%E8%83%B8&enc=utf-8&page={}".format(page_num))
#         time.sleep(3)
#         self.scroll(driver)
#         time.sleep(3)
#
#         # print driver.find_element_by_tag_name("a").text
#         print(driver.find_element_by_xpath(".//*[@id='J_goodsList']/ul/li/div/div[4]").text)
#         print(driver.find_element_by_xpath(".//*[@id='J_goodsList']/ul/li[60]/div/div[4]").text)
#         raw_context = driver.page_source
#         # print raw_context
#         root = etree.HTML(raw_context)
#         bra_list = root.xpath(".//*[@id='J_goodsList']/ul/li/div/div[4]/a/@href")
#         # print len(bra_list)
#         # for i in bra_list:
#         #     print i
#         driver.quit()
#         return bra_list
#
#     ##########################################################
#     ##########################################################
#
#     def process_request(self, request, spider):
#         if spider.name == "jd_item_spider":
#             print("正在使用js引擎爬取页面: PhantomJS is starting...")
#             driver = webdriver.PhantomJS(executable_path="D:\\phantomjs-1.9.7-windows\\phantomjs.exe") #指定使用的浏览器
#             try:
#                 driver.get(request.url)
#                 time.sleep(2)
#                 self.scroll(driver)
#                 time.sleep(4)
#                 body = driver.page_source
#                 print("访问了:" + request.url)
#
#                 ###########################################################################
#                 ###########################################################################
#                 resp = HtmlResponse(driver.current_url, body=body, encoding='utf-8', request=request)
#                 sel = Selector(response=resp)
#                 bra_list_tmp = sel.xpath(".//*[@id='J_goodsList']/ul/li/div/div[4]/a/@href").extract()
#                 print(bra_list_tmp)
#                 print(len(bra_list_tmp),
#                       "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++")
#                 bra_list = []
#                 for url in bra_list_tmp:
#                     url_deal = str(url).replace("//", "")
#                     bra_list.append(url_deal)
#
#                 for url in bra_list:
#                     config = {
#                         'host': "127.0.0.1"
#                         , 'port': 3306
#                         , 'user': 'root'
#                         , 'password': ''
#                         , 'db': 'test'
#                         , 'charset': 'utf8'
#                         , 'cursorclass': pymysql.cursors.DictCursor
#                     }
#                     connection = pymysql.connect(**config)
#                     try:
#                         with connection.cursor() as cursor:
#                             # 执行sql语句，插入记录
#                             sql = 'INSERT INTO jd_bra_url (url) VALUES (%s)'
#
#                             cursor.execute(sql, (url))
#                         # 没有设置默认自动提交，需要主动提交，以保存所执行的语句
#                         connection.commit()
#                     finally:
#                         connection.close()
#
#
#                 ###########################################################################
#                 ###########################################################################
#             except Exception as e:
#                 print("连接错误或者超时:{}".format(e[0]))
#             finally:
#                 driver.quit()
#
#             return resp
#         else:
#             return None



########################################################################################################################
########################################################################################################################

class CommentMiddleware():
    # 定义鼠标滚动到页面最底端的js函数
    def scroll(self,driver):
        driver.execute_script("""
            (function () {
                var y = document.body.scrollTop;
                var step = 100;
                window.scroll(0, y);
                function f() {
                    if (y < document.body.scrollHeight) {
                        y += step;
                        window.scroll(0, y);
                        setTimeout(f, 50);
                    }
                    else {
                        window.scroll(0, y);
                        document.title += "scroll-done";
                    }
                }
                setTimeout(f, 1000);
            })();
            """)

    """
    注意: 由于京东的页面滚动到最下面的时候,其实是加载了两页的内容
          所以,page=1和page=2其实是一样的内容
          文胸分类上点击下一页其实是跳转到page+2的url
          page_num取值范围[1,400]之间的偶数或者奇数
    """
    from scrapy.http import HtmlResponse

    def get_bralist_by_page(self,page_num):
        driver = webdriver.PhantomJS(executable_path="D:\\phantomjs-1.9.7-windows\\phantomjs.exe")
        driver.set_page_load_timeout(30)
        driver.get("https://search.jd.com/Search?keyword=%E6%96%87%E8%83%B8&enc=utf-8&page={}".format(page_num))
        time.sleep(3)
        self.scroll(driver)
        time.sleep(3)

        # print driver.find_element_by_tag_name("a").text
        print(driver.find_element_by_xpath(".//*[@id='J_goodsList']/ul/li/div/div[4]").text)
        print(driver.find_element_by_xpath(".//*[@id='J_goodsList']/ul/li[60]/div/div[4]").text)
        raw_context = driver.page_source
        # print raw_context
        root = etree.HTML(raw_context)
        bra_list = root.xpath(".//*[@id='J_goodsList']/ul/li/div/div[4]/a/@href")
        # print len(bra_list)
        # for i in bra_list:
        #     print i
        driver.quit()
        return bra_list

    ##########################################################
    ##########################################################

    def process_request(self, request, spider):
        if spider.name == "jd_item_spider":
            print("正在使用js引擎爬取页面: PhantomJS is starting...")
            driver = webdriver.PhantomJS(executable_path="D:\\phantomjs-1.9.7-windows\\phantomjs.exe") #指定使用的浏览器
            try:
                driver.get(request.url)
                time.sleep(2)
                self.scroll(driver)
                time.sleep(4)
                body = driver.page_source
                print("访问了:" + request.url)
                ###########################################################################
                ###########################################################################
                resp = HtmlResponse(driver.current_url, body=body, encoding='utf-8', request=request)
                return resp
                ###########################################################################
                ###########################################################################
            except Exception as e:
                print("连接错误或者超时:{}".format(e[0]))
            finally:
                driver.quit()

            return resp
        else:
            return None






